import random
import scrapy
from scrapy import signals
import requests
from scrapy.downloadermiddlewares.retry import RetryMiddleware


class RandomUserAgentMiddleware:
    """随机UserAgent"""

    def __init__(self, settings):
        self.settings = settings
        user_agent_file = settings.get('USER_AGENT_PATH')
        with open(user_agent_file, 'r') as f:
            self.user_agent_list = [i.strip() for i in f.readlines()]

    @classmethod
    def from_crawler(cls, crawler):
        # 会调用到 __init__ 方法上
        o = cls(crawler.settings)
        return o

    def process_request(self, request, spider):
        user_agent = random.choice(self.user_agent_list)
        request.headers['User-Agent'] = user_agent


class ProxyMiddleware:

    def __init__(self, proxy_url):
        self.proxy_url = proxy_url
        self.session = requests.session()

    @classmethod
    def from_crawler(cls, crawler):
        proxy_url = crawler.settings.get('PROXY_URL')
        return cls(proxy_url)

    def process_request(self, request, spider):
        resp = self.session.get(self.proxy_url, timeout=5)
        proxy = resp.json().get('proxy')
        proxy = f'http://{proxy}'
        # proxy = 'http://127.0.0.1:6666'
        request.meta['proxy'] = proxy


class MyRetryMiddleware(RetryMiddleware):
    def __init__(self, settings):
        # 调用父类__init__
        super(MyRetryMiddleware, self).__init__(settings)
        self.session = requests.session()
        self.proxy_url = settings.get('PROXY_URL')
        self.session = requests.session()

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings)

    def process_exception(self, request, exception, spider):
        if (
                isinstance(exception, self.EXCEPTIONS_TO_RETRY)
                and not request.meta.get('dont_retry', False)
        ):
            proxy = request.meta.get('proxy').replace('http://','')
            self.session.delete(self.proxy_url, params={'proxy': proxy})
            return self._retry(request, exception, spider)
